Part 1: Histogram of Oriented Gradients
Import Libraries
In [1]:
import matplotlib.image as mpimg
import matplotlib.pyplot as plt
import numpy as np
import cv2
import glob
from skimage.feature import hog
from sklearn.preprocessing import StandardScaler
import time
from sklearn.model_selection import train_test_split
from sklearn.svm import LinearSVC
import pickle
from scipy.ndimage.measurements import label
from collections import deque
#from skimage import color, exposure
# images are divided up into vehicles and non-vehicles
%matplotlib inline
Data Exploration
In [2]:
cars = glob.glob('vehicles/vehicles/**/*.png')
notcars = glob.glob('non-vehicles/non-vehicles/**/*.png')
# Define a function to return some characteristics of the dataset
def data_look(car_list, notcar_list):
data_dict = {}
# Define a key in data_dict "n_cars" and store the number of car images
data_dict["n_cars"] = len(car_list)
# Define a key "n_notcars" and store the number of notcar images
data_dict["n_notcars"] = len(notcar_list)
# Read in a test image, either car or notcar
example_img = mpimg.imread(car_list[0])
# Define a key "image_shape" and store the test image shape 3-tuple
data_dict["image_shape"] = example_img.shape
# Define a key "data_type" and store the data type of the test image.
data_dict["data_type"] = example_img.dtype
# Return data_dict
return data_dict
data_info = data_look(cars, notcars)
print('Your function returned a count of',
data_info["n_cars"], ' cars and',
data_info["n_notcars"], ' non-cars')
print('of size: ',data_info["image_shape"], ' and data type:',
data_info["data_type"])
# Just for fun choose random car / not-car indices and plot example images
car_ind = np.random.randint(0, len(cars))
notcar_ind = np.random.randint(0, len(notcars))
# Read in car / not-car images
car_image = mpimg.imread(cars[car_ind])
notcar_image = mpimg.imread(notcars[notcar_ind])
# Plot the examples
fig = plt.figure()
plt.subplot(121)
plt.imshow(car_image)
plt.title('Example Car Image')
plt.subplot(122)
plt.imshow(notcar_image)
plt.title('Example Not-car Image')
Out[2]:
A method using scikit-image HOG
In [3]:
# Define a function to return HOG features and visualization
def get_hog_features(img, orient, pix_per_cell, cell_per_block, vis=False, feature_vec=True):
if vis == True:
features, hog_image = hog(img, orientations=orient, pixels_per_cell=(pix_per_cell, pix_per_cell),
cells_per_block=(cell_per_block, cell_per_block), transform_sqrt=False,
visualise=True, feature_vector=False)
return features, hog_image
else:
features = hog(img, orientations=orient, pixels_per_cell=(pix_per_cell, pix_per_cell),
cells_per_block=(cell_per_block, cell_per_block), transform_sqrt=False,
visualise=False, feature_vector=feature_vec)
return features
# Define HOG parameters
orient = 9
pix_per_cell = 8
cell_per_block = 2
# Generate a random index to look at a car image
ind_car = np.random.randint(0, len(cars))
# Read in the image
image_car = mpimg.imread(cars[ind_car])
gray_car = cv2.cvtColor(image_car, cv2.COLOR_RGB2GRAY)
# Call our function with vis=True to see an image output
features_car, hog_image_car = get_hog_features(gray_car, orient,
pix_per_cell, cell_per_block,
vis=True, feature_vec=False)
# Generate a random index to look at a not car image
ind_notcar = np.random.randint(0, len(notcars))
# Read in the image
image_notcar = mpimg.imread(notcars[ind_notcar])
gray_notcar = cv2.cvtColor(image_notcar, cv2.COLOR_RGB2GRAY)
# Call our function with vis=True to see an image output
features_notcar, hog_image_notcar = get_hog_features(gray_notcar, orient,
pix_per_cell, cell_per_block,
vis=True, feature_vec=False)
# Plot the examples
f, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(6,6))
ax1.imshow(image_car)
ax1.set_title('Example Car Image')
ax2.imshow(hog_image_car, cmap='gray')
ax2.set_title('Car HOG Visualization')
ax3.imshow(image_notcar)
ax3.set_title('Example Non-Car Image')
ax4.imshow(hog_image_notcar, cmap='gray')
ax4.set_title('Non-Car HOG Visualization')
plt.tight_layout()
plt.show()
Extract Features Function
In [4]:
# Define a function to compute binned color features
def bin_spatial(img, size=(32, 32)):
# Use cv2.resize().ravel() to create the feature vector
features = cv2.resize(img, size).ravel()
# Return the feature vector
return features
# Define a function to compute color histogram features
def color_hist(img, nbins=32, bins_range=(0, 256)):
# Compute the histogram of the color channels separately
channel1_hist = np.histogram(img[:,:,0], bins=nbins, range=bins_range)
channel2_hist = np.histogram(img[:,:,1], bins=nbins, range=bins_range)
channel3_hist = np.histogram(img[:,:,2], bins=nbins, range=bins_range)
# Concatenate the histograms into a single feature vector
hist_features = np.concatenate((channel1_hist[0], channel2_hist[0], channel3_hist[0]))
# Return the individual histograms, bin_centers and feature vector
return hist_features
# Define a function to extract features from a list of images
# Have this function call bin_spatial() and color_hist()
def extract_features_color(imgs, cspace='RGB', spatial_size=(32, 32),
hist_bins=32, hist_range=(0, 256)):
# Create a list to append feature vectors to
features = []
# Iterate through the list of images
for file in imgs:
# Read in each one by one
image = mpimg.imread(file)
# apply color conversion if other than 'RGB'
if cspace != 'RGB':
if cspace == 'HSV':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2HSV)
elif cspace == 'LUV':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2LUV)
elif cspace == 'HLS':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2HLS)
elif cspace == 'YUV':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2YUV)
else: feature_image = np.copy(image)
# Apply bin_spatial() to get spatial color features
spatial_features = bin_spatial(feature_image, size=spatial_size)
# Apply color_hist() also with a color space option now
hist_features = color_hist(feature_image, nbins=hist_bins, bins_range=hist_range)
# Append the new feature vector to the features list
features.append(np.concatenate((spatial_features, hist_features)))
# Return list of feature vectors
return features
# Define a function to extract features from a list of images using HOG
def extract_features_hog(imgs, cspace='RGB', orient=9,
pix_per_cell=8, cell_per_block=2, hog_channel=0):
# Create a list to append feature vectors to
features = []
# Iterate through the list of images
for file in imgs:
# Read in each one by one
image = mpimg.imread(file)
# apply color conversion if other than 'RGB'
if cspace != 'RGB':
if cspace == 'HSV':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2HSV)
elif cspace == 'LUV':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2LUV)
elif cspace == 'HLS':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2HLS)
elif cspace == 'YUV':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2YUV)
elif cspace == 'YCrCb':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2YCrCb)
else: feature_image = np.copy(image)
# Call get_hog_features() with vis=False, feature_vec=True
if hog_channel == 'ALL':
hog_features = []
for channel in range(feature_image.shape[2]):
hog_features.append(get_hog_features(feature_image[:,:,channel],
orient, pix_per_cell, cell_per_block,
vis=False, feature_vec=True))
hog_features = np.ravel(hog_features)
else:
hog_features = get_hog_features(feature_image[:,:,hog_channel], orient,
pix_per_cell, cell_per_block, vis=False, feature_vec=True)
# Append the new feature vector to the features list
features.append(hog_features)
# Return list of feature vectors
return features
# Define a function to extract features from a list of images
def extract_features(imgs, cspace='RGB', orient=9,pix_per_cell=8, cell_per_block=2, hog_channel=0, spatial_size=(32, 32),
hist_bins=32, hist_range=(0, 256)):
# Create a list to append feature vectors to
features = []
# Iterate through the list of images
for file in imgs:
# Read in each one by one
image = mpimg.imread(file)
# apply color conversion if other than 'RGB'
if cspace != 'RGB':
if cspace == 'HSV':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2HSV)
elif cspace == 'LUV':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2LUV)
elif cspace == 'HLS':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2HLS)
elif cspace == 'YUV':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2YUV)
elif cspace == 'YCrCb':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2YCrCb)
else: feature_image = np.copy(image)
# Call get_hog_features() with vis=False, feature_vec=True
if hog_channel == 'ALL':
hog_features = []
for channel in range(feature_image.shape[2]):
hog_features.append(get_hog_features(feature_image[:,:,channel],
orient, pix_per_cell, cell_per_block,
vis=False, feature_vec=True))
hog_features = np.ravel(hog_features)
else:
hog_features = get_hog_features(feature_image[:,:,hog_channel], orient,
pix_per_cell, cell_per_block, vis=False, feature_vec=True)
# Apply bin_spatial() to get spatial color features
spatial_features = bin_spatial(feature_image, size=spatial_size)
# Apply color_hist() also with a color space option now
hist_features = color_hist(feature_image, nbins=hist_bins, bins_range=hist_range)
# Append the new feature vector to the features list
features.append(np.concatenate((spatial_features, hist_features, hog_features)))
return features
Extract Features and Data Preparation
In [5]:
# Feature extraction parameters
colorspace = 'YUV' # Can be RGB, HSV, LUV, HLS, YUV, YCrCb
orient = 11
pix_per_cell = 16
cell_per_block = 2
hog_channel = 'ALL' # Can be 0, 1, 2, or "ALL"
t = time.time()
car_features = extract_features_hog(cars, cspace=colorspace, orient=orient,
pix_per_cell=pix_per_cell, cell_per_block=cell_per_block,
hog_channel=hog_channel)
notcar_features = extract_features_hog(notcars, cspace=colorspace, orient=orient,
pix_per_cell=pix_per_cell, cell_per_block=cell_per_block,
hog_channel=hog_channel)
t2 = time.time()
print(round(t2-t, 2), 'Seconds to extract HOG features...')
# Create an array stack of feature vectors
X = np.vstack((car_features, notcar_features)).astype(np.float64)
# Fit a per-column scaler
X_scaler = StandardScaler().fit(X)
# Apply the scaler to X
scaled_X = X_scaler.transform(X)
# Define the labels vector
y = np.hstack((np.ones(len(car_features)), np.zeros(len(notcar_features))))
# Split up data into randomized training and test sets
rand_state = np.random.randint(0, 100)
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.2, random_state=rand_state)
print('Using:',orient,'orientations',pix_per_cell,
'pixels per cell and', cell_per_block,'cells per block')
print('Feature vector length:', len(X_train[0]))
Train a Classifier
In [6]:
# Use a linear SVC
svc = LinearSVC()
# Check the training time for the SVC
t = time.time()
svc.fit(X_train, y_train)
t2 = time.time()
print(round(t2-t, 2), 'Seconds to train SVC...')
# Check the score of the SVC
print('Test Accuracy of SVC = ', round(svc.score(X_test, y_test), 4))
# Check the prediction time for a single sample
t=time.time()
n_predict = 10
print('My SVC predicts: ', svc.predict(X_test[0:n_predict]))
print('For these',n_predict, 'labels: ', y_test[0:n_predict])
t2 = time.time()
print(round(t2-t, 5), 'Seconds to predict', n_predict,'labels with SVC')
Save the Data
In [7]:
# Save the train and test data
with open('trian_test.p', 'wb') as pfile:
pickle.dump(
{
'X_train': X_train,
'X_test': X_test,
'y_train': y_train,
'y_test': y_test
},
pfile, pickle.HIGHEST_PROTOCOL)
# Save the classifier model
with open('classifier.p', 'wb') as pfile:
pickle.dump(
{ 'svc':svc
},
pfile, pickle.HIGHEST_PROTOCOL)
print('Data have been saved!')
Part 2: Sliding Window Search
Load the classifier model
In [8]:
with open('classifier.p', mode='rb') as f:
data = pickle.load(f)
svc = data['svc']
Hog Sub-sampling Window Search
In [9]:
# Define a single function that can extract features using hog sub-sampling and make predictions
def find_cars(img, ystart, ystop, scale, cspace, hog_channel, svc, X_scaler, orient,
pix_per_cell, cell_per_block, all_rectangels=False):
# array of rectangles where cars were detected
rectangles = []
rectangles_all = []
draw_img = np.copy(img)
draw_all_rectangle = np.copy(img)
img = img.astype(np.float32)/255
img_tosearch = img[ystart:ystop,:,:]
# apply color conversion if other than 'RGB'
if cspace != 'RGB':
if cspace == 'HSV':
ctrans_tosearch = cv2.cvtColor(img_tosearch, cv2.COLOR_RGB2HSV)
elif cspace == 'LUV':
ctrans_tosearch = cv2.cvtColor(img_tosearch, cv2.COLOR_RGB2LUV)
elif cspace == 'HLS':
ctrans_tosearch = cv2.cvtColor(img_tosearch, cv2.COLOR_RGB2HLS)
elif cspace == 'YUV':
ctrans_tosearch = cv2.cvtColor(img_tosearch, cv2.COLOR_RGB2YUV)
elif cspace == 'YCrCb':
ctrans_tosearch = cv2.cvtColor(img_tosearch, cv2.COLOR_RGB2YCrCb)
else: ctrans_tosearch = np.copy(image)
# rescale image if other than 1.0 scale
if scale != 1:
imshape = ctrans_tosearch.shape
ctrans_tosearch = cv2.resize(ctrans_tosearch, (np.int(imshape[1]/scale), np.int(imshape[0]/scale)))
# select colorspace channel for HOG
if hog_channel == 'ALL':
ch1 = ctrans_tosearch[:,:,0]
ch2 = ctrans_tosearch[:,:,1]
ch3 = ctrans_tosearch[:,:,2]
else:
ch1 = ctrans_tosearch[:,:,hog_channel]
# Define blocks and steps as above
nxblocks = (ch1.shape[1] // pix_per_cell)+1 #-1
nyblocks = (ch1.shape[0] // pix_per_cell)+1 #-1
nfeat_per_block = orient*cell_per_block**2
# 64 was the orginal sampling rate, with 8 cells and 8 pix per cell
window = 64
nblocks_per_window = (window // pix_per_cell)-1
cells_per_step = 2 # Instead of overlap, define how many cells to step
nxsteps = (nxblocks - nblocks_per_window) // cells_per_step
nysteps = (nyblocks - nblocks_per_window) // cells_per_step
# Compute individual channel HOG features for the entire image
hog1 = get_hog_features(ch1, orient, pix_per_cell, cell_per_block, feature_vec=False)
if hog_channel == 'ALL':
hog2 = get_hog_features(ch2, orient, pix_per_cell, cell_per_block, feature_vec=False)
hog3 = get_hog_features(ch3, orient, pix_per_cell, cell_per_block, feature_vec=False)
for xb in range(nxsteps):
for yb in range(nysteps):
ypos = yb*cells_per_step
xpos = xb*cells_per_step
# Extract HOG for this patch
hog_feat1 = hog1[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel()
if hog_channel == 'ALL':
hog_feat2 = hog2[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel()
hog_feat3 = hog3[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel()
hog_features = np.hstack((hog_feat1, hog_feat2, hog_feat3))
else:
hog_features = hog_feat1
xleft = xpos*pix_per_cell
ytop = ypos*pix_per_cell
test_prediction = svc.predict([hog_features])
if test_prediction == 1:
xbox_left = np.int(xleft*scale)
ytop_draw = np.int(ytop*scale)
win_draw = np.int(window*scale)
cv2.rectangle(draw_img,(xbox_left, ytop_draw+ystart),(xbox_left+win_draw,ytop_draw+win_draw+ystart),(0,0,255),6)
rectangles.append(((xbox_left, ytop_draw+ystart),(xbox_left+win_draw,ytop_draw+win_draw+ystart)))
if all_rectangels is True:
xbox_left = np.int(xleft*scale)
ytop_draw = np.int(ytop*scale)
win_draw = np.int(window*scale)
color = (np.random.randint(0,255), np.random.randint(0,255), np.random.randint(0,255))
#color = (0, 0, 255)
cv2.rectangle(draw_all_rectangle,(xbox_left, ytop_draw+ystart),(xbox_left+win_draw,ytop_draw+win_draw+ystart),color,6)
rectangles_all.append(((xbox_left, ytop_draw+ystart),(xbox_left+win_draw,ytop_draw+win_draw+ystart)))
return rectangles, rectangles_all, draw_img, draw_all_rectangle
# Test the find_car() function
def test_find_cars(img_path, ystart, ytstop, scale):
img = mpimg.imread(img_path)
ystart = ystart
ystop = ytstop
scale = scale
# Feature extraction parameters
colorspace = 'YUV' # Can be RGB, HSV, LUV, HLS, YUV, YCrCb
orient = 11
pix_per_cell = 16
cell_per_block = 2
hog_channel = 'ALL' # Can be 0, 1, 2, or "ALL"
rectangles, _, out_img, all_img= find_cars(img, ystart, ystop, scale, colorspace, hog_channel, svc, None, orient, pix_per_cell, cell_per_block, all_rectangels=True)
print(len(rectangles), 'rectangles found in test image')
plt.figure()
# Plot the result
f, (ax1,ax2) = plt.subplots(1, 2, figsize=(24, 9))
f.tight_layout()
ax1.imshow(all_img)
ax1.set_title('All windows', fontsize=40)
ax2.imshow(out_img)
ax2.set_title('Detected windows', fontsize=40)
plt.show()
test_find_cars('./test_images/test1.jpg', 400, 656, 1.5)
In [10]:
### Test the find cars on all the test images
test_images = glob.glob('./test_images/test*.jpg')
fig, axs = plt.subplots(3, 2, figsize=(16,14))
fig.subplots_adjust(hspace = .004, wspace=.002)
axs = axs.ravel()
for i, im in enumerate(test_images):
img = mpimg.imread(im)
ystart = 400
ystop = 656
scale = 1.5
# Feature extraction parameters
colorspace = 'YUV' # Can be RGB, HSV, LUV, HLS, YUV, YCrCb
orient = 11
pix_per_cell = 16
cell_per_block = 2
hog_channel = 'ALL' # Can be 0, 1, 2, or "ALL"
_, _, out_img, _= find_cars(img, ystart, ystop, scale, colorspace, hog_channel, svc, None, orient, pix_per_cell, cell_per_block, all_rectangels=False)
axs[i].imshow(out_img)
axs[i].set_title('test'+str(i+1)+'.jpg', fontsize=20)
axs[i].axis('off')
# Show all the images in the figure
plt.tight_layout()
plt.show()
Search in different scales
As the udacity lesson described, Each window is defined by a scaling factor where a scale of 1 would result in a window that's 8 x 8 cells then the overlap of each window is in terms of the cell distance. 1 cell is 8 x 8 pixels. So we can use different scale based on the y pixels of 1 window: 1 scale ->64 pixels; 1.5 scale -> 96 pixels; 2 scale-> 128 pixels; 2.5 scale -> 160 pixels; 3 scale -> 192 pixels; 3.5 scale -> 224 pixels.
This means that a cells_per_step = 2 would result in a search window overlap of 75%, so if you want to find a cell of next row , you shuold move down 15% scale pixels(ex: 1 scale moved down 16 pixels to make a new cell.)
In order to search efficiently and not to find a lot duplicate region, I wil define differnt scale only on searching 2 rows:
- 1 scale: (start:400, stop:464) ; (start:416, stop:480)
- 1.5 scale: (start: 400, stop:496); (start:424, stop:520)
- 2 scale:(start: 400, stop:528); (start:432, stop:560)
- 2.5 scale(start: 400, stop:560); (start:440, stop:600)
- 3 scale(start:400, stop:592); (start:448, stop:640)
- 3.5 scale(start:400, stop:624);(start:456, stop:680)
In [11]:
def find_all_scales(img, draw_rectangles=False):
# Rectangle list to store different scales window
rectangles_list = []
# Feature extraction parameters
colorspace = 'YUV' # Can be RGB, HSV, LUV, HLS, YUV, YCrCb
orient = 11
pix_per_cell = 16
cell_per_block = 2
hog_channel = 'ALL' # Can be 0, 1, 2, or "ALL"
# Define which the scales list
scale_list = [1.0, 1.5, 2.0, 2.5, 3.0, 3.5]
# Save diffent y range and scaled winow
for scale in scale_list:
if scale == 1.0:
rectangles,_, _, _= find_cars(img, 400, 464, scale, colorspace, hog_channel, svc, None, orient, pix_per_cell, cell_per_block, all_rectangels=False)
rectangles_list.append(rectangles)
rectangles,_, _, _= find_cars(img, 416, 480, scale, colorspace, hog_channel, svc, None, orient, pix_per_cell, cell_per_block, all_rectangels=False)
rectangles_list.append(rectangles)
elif scale == 1.5:
rectangles,_, _, _= find_cars(img, 400, 496, scale, colorspace, hog_channel, svc, None, orient, pix_per_cell, cell_per_block, all_rectangels=False)
rectangles_list.append(rectangles)
rectangles,_, _, _= find_cars(img, 424, 520, scale, colorspace, hog_channel, svc, None, orient, pix_per_cell, cell_per_block, all_rectangels=False)
rectangles_list.append(rectangles)
elif scale == 2.0:
rectangles,_, _, _= find_cars(img, 400, 528, scale, colorspace, hog_channel, svc, None, orient, pix_per_cell, cell_per_block, all_rectangels=False)
rectangles_list.append(rectangles)
rectangles,_, _, _= find_cars(img, 432, 560, scale, colorspace, hog_channel, svc, None, orient, pix_per_cell, cell_per_block, all_rectangels=False)
rectangles_list.append(rectangles)
elif scale == 3.5:
rectangles, _, _,_= find_cars(img, 400, 624, scale, colorspace, hog_channel, svc, None, orient, pix_per_cell, cell_per_block, all_rectangels=False)
rectangles_list.append(rectangles)
rectangles, _, _,_= find_cars(img, 456, 680, scale, colorspace, hog_channel, svc, None, orient, pix_per_cell, cell_per_block, all_rectangels=False)
rectangles_list.append(rectangles)
"""
elif scale == 2.5:
rectangles,_, _, _= find_cars(img, 400, 560, scale, colorspace, hog_channel, svc, None, orient, pix_per_cell, cell_per_block, all_rectangels=False)
rectangles_list.append(rectangles)
rectangles,_, _, _= find_cars(img, 440, 600, scale, colorspace, hog_channel, svc, None, orient, pix_per_cell, cell_per_block, all_rectangels=False)
rectangles_list.append(rectangles)
elif scale == 3.0:
rectangles,_, _, _= find_cars(img, 400, 592, scale, colorspace, hog_channel, svc, None, orient, pix_per_cell, cell_per_block, all_rectangels=False)
rectangles_list.append(rectangles)
rectangles,_, _, _= find_cars(img, 448, 656, scale, colorspace, hog_channel, svc, None, orient, pix_per_cell, cell_per_block, all_rectangels=False)
rectangles_list.append(rectangles)
"""
rectangles = [item for sublist in rectangles_list for item in sublist]
# Draw boxes
# Make a copy of the image
imcopy = np.copy(img)
if draw_rectangles is True:
# Iterate through the bounding boxes
for bbox in rectangles:
color = (np.random.randint(0,255), np.random.randint(0,255), np.random.randint(0,255))
# Draw a rectangle given bbox coordinates
cv2.rectangle(imcopy, bbox[0], bbox[1], color, 6)
return imcopy, rectangles
test_img_rectangles, rectangles = find_all_scales(mpimg.imread('./test_images/test1.jpg'), draw_rectangles=True)
plt.figure(figsize=(10,15))
plt.imshow(test_img_rectangles)
Out[11]:
In [12]:
### Test the find all scales on all the test images
test_images = glob.glob('./test_images/test*.jpg')
fig, axs = plt.subplots(3, 2, figsize=(16,14))
fig.subplots_adjust(hspace = .004, wspace=.002)
axs = axs.ravel()
for i, im in enumerate(test_images):
img = mpimg.imread(im)
out_img, _ = find_all_scales(img, draw_rectangles=True)
axs[i].imshow(out_img)
axs[i].set_title('test'+str(i+1)+'.jpg', fontsize=20)
axs[i].axis('off')
# Show all the images in the figure
plt.tight_layout()
plt.show()
In [13]:
### Draw all scales on one image
def draw_all_scales(img):
# Rectangle list to store different scales window
rectangles_list = []
# Feature extraction parameters
colorspace = 'YUV' # Can be RGB, HSV, LUV, HLS, YUV, YCrCb
orient = 11
pix_per_cell = 16
cell_per_block = 2
hog_channel = 'ALL' # Can be 0, 1, 2, or "ALL"
# Define which the scales list
scale_list = [1.0, 1.5, 2.0, 2.5, 3.0, 3.5]
# Save diffent y range and scaled winow
for scale in scale_list:
if scale == 1.0:
_,rectangles,_, _= find_cars(img, 400, 464, scale, colorspace, hog_channel, svc, None, orient, pix_per_cell, cell_per_block, all_rectangels=True)
rectangles_list.append(rectangles)
_,rectangles,_, _= find_cars(img, 416, 480, scale, colorspace, hog_channel, svc, None, orient, pix_per_cell, cell_per_block, all_rectangels=True)
rectangles_list.append(rectangles)
elif scale == 1.5:
_,rectangles,_, _= find_cars(img, 400, 496, scale, colorspace, hog_channel, svc, None, orient, pix_per_cell, cell_per_block, all_rectangels=True)
rectangles_list.append(rectangles)
_,rectangles,_, _= find_cars(img, 424, 520, scale, colorspace, hog_channel, svc, None, orient, pix_per_cell, cell_per_block, all_rectangels=True)
rectangles_list.append(rectangles)
elif scale == 2.0:
_,rectangles,_, _= find_cars(img, 400, 528, scale, colorspace, hog_channel, svc, None, orient, pix_per_cell, cell_per_block, all_rectangels=True)
rectangles_list.append(rectangles)
_,rectangles,_, _= find_cars(img, 432, 560, scale, colorspace, hog_channel, svc, None, orient, pix_per_cell, cell_per_block, all_rectangels=True)
rectangles_list.append(rectangles)
elif scale == 3.5:
_,rectangles, _, _= find_cars(img, 400, 624, scale, colorspace, hog_channel, svc, None, orient, pix_per_cell, cell_per_block, all_rectangels=True)
rectangles_list.append(rectangles)
_,rectangles, _, _= find_cars(img, 456, 680, scale, colorspace, hog_channel, svc, None, orient, pix_per_cell, cell_per_block, all_rectangels=True)
rectangles_list.append(rectangles)
"""
elif scale == 2.5:
_,rectangles,_, _= find_cars(img, 400, 560, scale, colorspace, hog_channel, svc, None, orient, pix_per_cell, cell_per_block, all_rectangels=True)
rectangles_list.append(rectangles)
_,rectangles,_, _= find_cars(img, 440, 600, scale, colorspace, hog_channel, svc, None, orient, pix_per_cell, cell_per_block, all_rectangels=True)
rectangles_list.append(rectangles)
elif scale == 3.0:
_,rectangles,_, _= find_cars(img, 400, 592, scale, colorspace, hog_channel, svc, None, orient, pix_per_cell, cell_per_block, all_rectangels=True)
rectangles_list.append(rectangles)
_,rectangles,_, _= find_cars(img, 448, 656, scale, colorspace, hog_channel, svc, None, orient, pix_per_cell, cell_per_block, all_rectangels=True)
rectangles_list.append(rectangles)
"""
rectangles = [item for sublist in rectangles_list for item in sublist]
print(len(rectangles), 'rectangles found.')
# Draw boxes
# Make a copy of the image
imcopy = np.copy(img)
random_color = False
# Iterate through the bounding boxes
for bbox in rectangles:
color = (np.random.randint(0,255), np.random.randint(0,255), np.random.randint(0,255))
# Draw a rectangle given bbox coordinates
cv2.rectangle(imcopy, bbox[0], bbox[1], color, 6)
plt.figure(figsize=(10,15))
plt.imshow(imcopy)
draw_all_scales(img)
False Positives using Heat Map
In [14]:
### Add heat map
def add_heat(heatmap, bbox_list):
# Iterate through list of bboxes
for box in bbox_list:
# Add += 1 for all pixels inside each bbox
# Assuming each "box" takes the form ((x1, y1), (x2, y2))
heatmap[box[0][1]:box[1][1], box[0][0]:box[1][0]] += 1
# Return updated heatmap
return heatmap
# Test the add heat map
heatmap_img = np.zeros_like(test_img_rectangles[:,:,0])
heatmap_img = add_heat(heatmap_img, rectangles)
plt.figure(figsize=(10,15))
plt.imshow(heatmap_img, cmap='hot')
Out[14]:
In [15]:
### Apply Threshold
def apply_threshold(heatmap, threshold):
# Zero out pixels below the threshold
heatmap[heatmap <= threshold] = 0
# Return thresholded map
return heatmap
# Test the apply_threshold
heatmap_img = apply_threshold(heatmap_img, 1)
plt.figure(figsize=(10,15))
plt.imshow(heatmap_img, cmap='hot')
Out[15]:
In [16]:
### Apply SciPy Labels
labels = label(heatmap_img)
print(labels[1], 'cars found')
plt.figure(figsize=(10,15))
plt.imshow(labels[0], cmap='gray')
Out[16]:
In [17]:
### Draw Bounding Boxes for Lables
def draw_labeled_bboxes(img, labels):
# Iterate through all detected cars
for car_number in range(1, labels[1]+1):
# Find pixels with each car_number label value
nonzero = (labels[0] == car_number).nonzero()
# Identify x and y values of those pixels
nonzeroy = np.array(nonzero[0])
nonzerox = np.array(nonzero[1])
# Define a bounding box based on min/max x and y
bbox = ((np.min(nonzerox), np.min(nonzeroy)), (np.max(nonzerox), np.max(nonzeroy)))
# Draw the box on the image
cv2.rectangle(img, bbox[0], bbox[1], (0,0,255), 6)
# Return the image
return img
# Draw bounding boxes on a copy of the image
draw_img = draw_labeled_bboxes(np.copy(img), labels)
# Display the image
plt.figure(figsize=(10,15))
plt.imshow(draw_img)
Out[17]:
Putting all previous steps to form a Pipline to process the frame image
In [18]:
def process_image(image):
# Search different scales window
_, rectangles = find_all_scales(image)
# Add heat map
heatmap_img = np.zeros_like(image[:,:,0])
heatmap_img = add_heat(heatmap_img, rectangles)
### Apply Threshold
heatmap_img = apply_threshold(heatmap_img, 1)
labels = label(heatmap_img)
### Draw Bounding Boxes for Lables
result = draw_labeled_bboxes(np.copy(image), labels)
return result
# Test the pipleline on all the test images
test_images = glob.glob('./test_images/test*.jpg')
fig, axs = plt.subplots(3, 2, figsize=(16,14))
fig.subplots_adjust(hspace = .004, wspace=.002)
axs = axs.ravel()
for i, im in enumerate(test_images):
axs[i].imshow(process_image(mpimg.imread(im)))
axs[i].set_title('test'+str(i+1)+'.jpg', fontsize=20)
axs[i].axis('off')
# Show all the images in the figure
plt.tight_layout()
plt.show()
Part 3: Video Implementation
Run Pipeline on test video
In [19]:
from moviepy.editor import VideoFileClip
from IPython.display import HTML
white_output = 'test_video_output.mp4'
clip1 = VideoFileClip("test_video.mp4")
white_clip = clip1.fl_image(process_image)
%time white_clip.write_videofile(white_output, audio=False)
HTML("""
<video width="960" height="540" controls>
<source src="{0}">
</video>
""".format(white_output))
Out[19]:
Refine the Pipeline to store the previous frames and select the interested region
In [20]:
# Applies an image mask.
def region_of_interest(img, vertices):
#defining a blank mask to start with
mask = np.zeros_like(img)
#defining a 3 channel or 1 channel color to fill the mask with depending on the input image
if len(img.shape) > 2:
channel_count = img.shape[2] # i.e. 3 or 4 depending on your image
ignore_mask_color = (255,) * channel_count
else:
ignore_mask_color = 255
#filling pixels inside the polygon defined by "vertices" with the fill color
cv2.fillPoly(mask, vertices, ignore_mask_color)
#returning the image only where mask pixels are nonzero
masked_image = cv2.bitwise_and(img, mask)
return masked_image
# Define a deque to store the previous frames
recent_frames = deque([], maxlen=10)
### Pipeline Refine
def process_image_refine(image):
# This time we are defining a four sided polygon to mask
img_mask = np.copy(image)
imshape = img_mask.shape
vertices = np.array([[(0,imshape[0]),(550,400), (imshape[1], 0), (imshape[1],imshape[0])]], dtype=np.int32)
img_mask = region_of_interest(img_mask, vertices)
# Search different scales window
_, rectangles = find_all_scales(img_mask)
global recent_frames
if len(rectangles) > 0:
if(len(recent_frames) < 10):
recent_frames.append(rectangles)
else:
recent_frames.popleft()
recent_frames.append(rectangles)
# Add heat map
heatmap_img = np.zeros_like(image[:,:,0])
for rec in recent_frames:
heatmap_img = add_heat(heatmap_img, rec)
### Apply Threshold
heatmap_img = apply_threshold(heatmap_img, len(recent_frames) // 2 + 1)
#heatmap_img = apply_threshold(heatmap_img, 4)
labels = label(heatmap_img)
### Draw Bounding Boxes for Lables
result = draw_labeled_bboxes(np.copy(image), labels)
return result
Run the refined Pinpline on the test video
In [21]:
from moviepy.editor import VideoFileClip
from IPython.display import HTML
# Clear the stroe deque
recent_frames.clear()
white_output = 'test_video_output_2.mp4'
clip1 = VideoFileClip("test_video.mp4")
white_clip = clip1.fl_image(process_image_refine)
%time white_clip.write_videofile(white_output, audio=False)
HTML("""
<video width="960" height="540" controls>
<source src="{0}">
</video>
""".format(white_output))
Out[21]:
Process the project video
In [22]:
from moviepy.editor import VideoFileClip
from IPython.display import HTML
# Clear the stroe deque
recent_frames.clear()
white_output = 'project_video_output.mp4'
clip1 = VideoFileClip("project_video.mp4")
white_clip = clip1.fl_image(process_image_refine)
%time white_clip.write_videofile(white_output, audio=False)
HTML("""
<video width="960" height="540" controls>
<source src="{0}">
</video>
""".format(white_output))
Out[22]:
In [ ]: